package cn.com.klun.tjy.javaSpider.testss;

import org.jsoup.*;
import org.jsoup.nodes.*;
import org.jsoup.select.*;

import java.io.*;
import java.net.URL;
import java.net.URLConnection;

public class Html1 {

    public static void Get_Url(String url) {
        try {
         Document doc = Jsoup.connect(url)
          //.data("query", "Java")
           .userAgent("Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; MALC)")
           //.cookie("auth", "token")
           //.timeout(3000)
           //.post()
           .get();

         //得到html的所有东西
         Element content = doc.getElementById("content");


//         //分离出html下<a>...</a>之间的所有东西
//         Elements links = content.getElementsByTag("a");
//         //Elements links = doc.select("a[href]");
//         // 扩展名为.png的图片
//         Elements pngs = doc.select("img[src$=.png]");
//         // class等于masthead的div标签
//         Element masthead = doc.select("div.masthead").first();
//
//         for (Element link : links) {
//               //得到<a>...</a>里面的网址
//               String linkHref = link.attr("href");
//               //得到<a>...</a>里面的汉字
//               String linkText = link.text();
//               System.out.println(linkText);
//         }



         } catch (IOException e) {
               e.printStackTrace();
         }
     }

    //将抓取的网页变成html文件，保存在本地
    public static void Save_Html(String url) {
        try {
            File dest = new File("src/temp_html/" + "保存的html的名字.html");
            //接收字节输入流
            InputStream is;
            //字节输出流
            FileOutputStream fos = new FileOutputStream(dest);
            URL temp = new URL(url);
            // 防止403错误
            URLConnection uc = temp.openConnection();
            uc.addRequestProperty("User-Agent", "Mozilla/5.0 (iPad; U; CPU OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5");

            is = temp.openStream();

            //为字节输入流加缓冲
            BufferedInputStream bis = new BufferedInputStream(is);
            //为字节输出流加缓冲
            BufferedOutputStream bos = new BufferedOutputStream(fos);

            int length;

            byte[] bytes = new byte[1024*20];
            while((length = bis.read(bytes, 0, bytes.length)) != -1){
                fos.write(bytes, 0, length);
            }

            bos.close();
            fos.close();
            bis.close();
            is.close();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}